//============================================================================
// Name        : Parser.h
// Author      : lucmir@dcc.ufmg.br
// Description :
//============================================================================

#ifndef __PARSER_H__
#define __PARSER_H__

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <sys/time.h>
#include <cstdlib>
#include <iostream>
#include <map>
#include <sstream>

#include "htmlcxx/html/ParserDom.h"
#include "htmlcxx/html/Node.h"
#include "htmlcxx/html/utils.h"
#include "htmlcxx/html/CharsetConverter.h"
#include "CollectionReader.h"
#include "Encoder.h"

#include "Run.h"
#include "Lexon.h"
#include "DocumentInfo.h"
#include "DocumentIndex.h"
#include "AnchorText.h"

#define TOKEN_DELIMITERS	" :,.;\n{}[]()*!@#$%<>=+\\\\/|=\r\n\t?'\"&`_-0123456789"


class Parser {

	RICPNS::CollectionReader *reader;
	RICPNS::Document doc;

	public:

		Parser(std::string inputDirectory, std::string indexFileName);
		~Parser();

		bool getNextDocument();

		/**
		 * Function to parse a document.
		 * New terms are added to the vocabulary and to a run.
		 */
		void parseDocument(Run *run, Lexon *lexon, DocumentIndex *docIdx,
			std::ofstream *docLinksFile, AnchorText *anchorText);

		/**
		 * Function to pre-process term string used in query
		 */
		static std::string preprocessQueryTerm(std::string str);
};

#endif
